﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;

namespace DotSpider.Controls
{
    public class Indexer
    {
        public static void IndexPage(string url, string status, MySQL db)
        {
            string content = GetWebContent(url);
            int full_count = content.Length;
            if (full_count > 0)
            {
                int pageId = AddPageInfoToDB(url, Settings.GrepPageTitle(content), full_count, db);
                List<string> links = GetAllLinks(url, content);
                AddLinksToQueue(links, db);
                CleanTags(ref content);
                int count = content.Length;
                AnalysData(pageId, content, db);
            }
            RemoveFromQueue(url, db);
        }

        private static void AddLinksToQueue(List<string> links, MySQL db)
        {
            Settings.Logger.AddLog(string.Format("Indexer > Adding '{0}' link to queue", links.Count));
            db.AddToQueue(links);
        }

        private static void RemoveFromQueue(string url, MySQL db)
        {
            db.RemoveQueue(url);
            Settings.Logger.AddLog("Indexer > Removing page '" + url + "'");
        }

        private static int AddPageInfoToDB(string url, string title, int contentSize, MySQL db)
        {
            return db.AddPageInfo(url, title, contentSize);
        }

        public static Dictionary<string, int> ExtractWords(string content)
        {
            Dictionary<string, int> pageWords = new Dictionary<string, int>();
            Regex rxWords = new Regex(@"[^\w]?([\d\w]+)[^\w]?", RegexOptions.Multiline);
            MatchCollection rxMatches = rxWords.Matches(content);
            foreach (Match match in rxMatches)
            {
                string word = match.Groups[1].Captures[0].ToString().ToLower(); //Without spaces to lower
                if (!pageWords.ContainsKey(word))
                {
                    pageWords.Add(word, 1);
                } else
                {
                    pageWords[word]++;
                }
            }
            return pageWords;
        }

        private static void AnalysData(int pageId, string content, MySQL db)
        {
            Dictionary<string, int> newWords = new Dictionary<string, int>();
            Dictionary<string, int> pageWords = ExtractWords(content);
            
            foreach (string word in pageWords.Keys)
            {
                if (!Settings.Words.ContainsKey(word) && !newWords.ContainsKey(word))
                {
                    newWords.Add(word, pageWords[word]);
                } 
            }
            
            Settings.Logger.AddLog(string.Format("Indexer > Adding '{0}' new words to db", newWords.Count));
            
            foreach (string word in newWords.Keys)
            {
                AddWordToDB(word, db);
            }

            AddPageToDB(pageId, pageWords, db);
        }

        private static void AddPageToDB(int pageId, Dictionary<string, int> pageWords, MySQL db)
        {
            int position = 1;
            foreach (string word in pageWords.Keys)
            {
                db.AddPageWord(pageId, position++, word, pageWords[word]);
            }
        }

        private static void AddWordToDB(string word, MySQL db)
        {
            int index = db.AddWord(word);
            if (!Settings.Words.ContainsKey(word))
                Settings.Words.Add(word, index);
        }

        private static List<string> GetAllLinks(string url, string content)
        {
            List<string> links = new List<string>();

            // Normal links
            Regex rxLinks = new Regex(@"href\=" + '"' + @"([^\" + '"' + @"]+)\" + '"', RegexOptions.IgnoreCase); // href\=\"([^\"]+)\"
            Regex oldLinks = new Regex(@"href=([^\" + '"' + @"][^\s>]+)>", RegexOptions.IgnoreCase);
            FindLinks(rxLinks, content, url, links);
            FindLinks(oldLinks, content, url, links);
            
            // Filter images and archivs, etc
            List<string> removeExt = new List<string>(new string[] 
                                { "rar", "zip", "arz", "jpg", "png", "gif" });
            foreach (string link in new List<string>(links))
            {
                foreach (string ext in removeExt)
                {
                    if (Regex.IsMatch(link, @"\." +ext+@"$",RegexOptions.IgnoreCase))
                    {
                        links.Remove(link);
                        break;
                    }
                }
            }

            return links;
        }

        private static void FindLinks(Regex rxLinks, string content, string url, List<string> links)
        {
            MatchCollection rxMatches = rxLinks.Matches(content);
            foreach (Match match in rxMatches)
            {
                string link = match.Groups[1].Captures[0].ToString(); //Without spaces to lower
                link = Settings.FixURL(url, link);
                if (!links.Contains(link) 
                    && link.IndexOf((new Uri(url)).Host) == 7) links.Add(link); //Dont change server
            }
        }

        private static void CleanTags(ref string content)
        {
            //Cut all not used for text tags
            List<string> tagFullRemoveList = new List<string>(new string[]{"script", "head", "style"});
            foreach (string s in tagFullRemoveList)
            {
                while (true)
                {
                    int start = content.IndexOf(@"<" + s,StringComparison.OrdinalIgnoreCase);
                    int end = content.IndexOf(@"</" + s + ">", StringComparison.OrdinalIgnoreCase);
                    if (end > start && end != -1 && start != -1)
                    {
                        string textToRemove = content.Substring(start, end - start + (3 + s.Length));
                        content = content.Replace(textToRemove, "");
                        continue;
                    }
                    break;
                }
            }
            //Spaces
            content = content.Replace("&nbsp;", " ");
            //Fully remove all tags
            Regex rxClean = new Regex(@"\<[^\>]+\>",RegexOptions.Multiline);
            content = rxClean.Replace(content, " ");
        }

        private static string GetWebContent(string url)
        {
            try
            {
                // Create the web request
                HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
                if (Settings.UseProxy())
                {
                    request.Proxy = Settings.GetProxy(true);
                }

                // Get response
                using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
                {
                    // Get the response stream

                    if (response.ContentType.IndexOf("text/html") < 0) return "";
                    //TODO: Remove 'IF', use 'Switch'.
                    StreamReader reader;
                    string encode = response.CharacterSet;
                    switch(encode)
                    {
                        case "utf-8":
                            reader = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
                            break;
                        case "windows-1251":
                        default:
                            reader = new StreamReader(response.GetResponseStream(), Encoding.Default);
                            break;
                    }
                    // Console application output
                    return reader.ReadToEnd();
                }
            }
            catch(Exception e)
            {
                //throw new Exception("Can't index sended URL: " + e.Message);
                Settings.Logger.AddLog("Error. Can't index sended URL: " + e.Message);
            }
            return "";
        }
    }
}
